package com.duowan.mspider.processor;

import com.duowan.mspider.util.Constant;
import com.duowan.mspider.util.Constant.KeyWordsFile;
import com.duowan.mspider.util.FileUtils;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

import java.io.File;
import java.util.List;
import java.util.Map;

/**
 * @author:wangyan1
 * @create_time: 2018-04-20 18:13
 */

public class DistinctFile {

	public static void main(String[] args) throws Exception {

		//文件去重
		Map<KeyWordsFile, List<String>> map = Maps.newHashMap();
		for (KeyWordsFile keyWordsFile : Constant.KeyWordsFile.values()) {
			List<String> result = Lists.newArrayList();

			//其他搜索引擎爬虫结果
			File tempFile = new File(keyWordsFile.getTempFile());
			add(result, tempFile);

			//神马引擎运行结果
			File smTempFile = new File(keyWordsFile.getSmTempFile());
			add(result, smTempFile);

			map.put(keyWordsFile, result);

			if (tempFile.exists()) tempFile.delete();
			if (smTempFile.exists()) tempFile.delete();
		}

		for (KeyWordsFile keyWordsFile : map.keySet()) {
			File destFile = new File(keyWordsFile.destFileName);
			if (destFile.exists()) destFile.delete();

			for (String str : map.get(keyWordsFile)) {
				FileUtils.write(destFile, str);
			}
		}
	}

	public static void add(List<String> result, File file) {
		if (file.exists()) {
			List<String> temp = FileUtils.read(file);
			for (String str : temp) {
				if (!result.contains(str)) result.add(str);
			}
		}
	}

}
